#!/usr/bin/python3
# -*- coding:utf-8 -*-
# yn-2364511934@qq.com

# 从拉钩网上爬取数据练习

from urllib.request import urlopen
from urllib.request import Request
from bs4 import  BeautifulSoup
from time import sleep
import pymysql
import re
import datetime

def get_date_time(str):
    return_str = None
    return_str = re.match("(1)+?(天)+(前)+?(发)+?(布)+?.*", str)
    if None != return_str:
        return (datetime.date.today() - datetime.timedelta(days=1))
    return_str = re.match("(2)+?(天)+(前)+?(发)+?(布)+?.*", str)
    if None != return_str:
        return (datetime.date.today() - datetime.timedelta(days=2))
    return_str = re.match("(3)+?(天)+(前)+?(发)+?(布)+?.*", str)
    if None != return_str:
        return (datetime.date.today() - datetime.timedelta(days=3))

    return_str = re.match("[0-9]+?[0-9]+?(:)[0-9]+[0-9]+?(发)+?(布)+?", str)
    if None != return_str:
        return datetime.date.today()
    return str

head={"User-Agent":'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36',
     "Cookie":'_ga=GA1.2.1616008838.1522242619; user_trace_token=20180328211027-627700ef-3289-11e8-b653-5254005c3644; LGUID=20180328211027-6277052b-3289-11e8-b653-5254005c3644; index_location_city=%E5%8C%97%E4%BA%AC; JSESSIONID=ABAAABAAAIAACBIF94425347F8A5214A48CA6D4FAA7A93F; _gat=1; LGSID=20180331092901-e45d8702-3482-11e8-a60d-525400f775ce; PRE_UTM=; PRE_HOST=; PRE_SITE=; PRE_LAND=https%3A%2F%2Fwww.lagou.com%2F; LGRID=20180331092901-e45d88d4-3482-11e8-a60d-525400f775ce; _gid=GA1.2.1066012269.1522459725; Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1522242619,1522282483,1522459726; Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1522459726'}
lagou_url="https://www.lagou.com/"
request = Request(lagou_url, headers=head)
url_response = urlopen(request)

bs = BeautifulSoup(url_response, "html.parser")
# 获取到全部的工作职位的基础链接
links_list= bs.select("div.menu_sub a")
print(len(links_list))
real_links_list = []
for link in links_list:
    lk = link.get("href").strip()
    real_links_list.append(lk)
#print(links_list)
#https://www.lagou.com/zhaopin/Java/2/

con = pymysql.connect(host="hadoop01", user="root", password="root", database="spider", charset='utf8', port=3306)
cursor = con.cursor()
lagou_insert_sql = "insert into lagou (t_job, t_addr, t_com, t_tag, t_money, t_edu, t_exp, t_time, t_pos_type, t_culture) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"

"""
CREATE TABLE `lagou` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `t_job` varchar(255) DEFAULT NULL COMMENT '招聘职位',
  `t_addr` varchar(255) DEFAULT NULL COMMENT '公司地址',
  `t_tag` varchar(255) DEFAULT NULL COMMENT '公司类型',
  `t_com` varchar(255) DEFAULT NULL COMMENT '公司名称',
  `t_money` varchar(255) DEFAULT NULL COMMENT '薪资',
  `t_edu` varchar(255) DEFAULT NULL COMMENT '学历',
  `t_exp` varchar(255) DEFAULT NULL COMMENT '经验',
  `t_time` varchar(255) DEFAULT NULL COMMENT '职位发布时间',
  `t_pos_type` varchar(255) DEFAULT NULL COMMENT '职位类型',
  `t_culture` varchar(255) DEFAULT NULL COMMENT '公司文化',
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
"""
def get_job_information(link):
    i=0
    while True:
        i += 1
        # print(link)
        sleep(2)
        job_link = link + str(i) + '/'
        print(job_link)
        request = Request(job_link, headers=head)
        link_response= urlopen(request)
        bs_job = BeautifulSoup(link_response, "html.parser")
        jobs_list = bs_job.select("div.page404")
        if 0 !=  len(jobs_list):
            return True
        jobs_list = bs_job.select("li.con_list_item")
        # if job_link != link_response.geturl():
        #     print("没有数据，继续抓取下一个链接")
        #     return False
        #print(jobs_list)
        # if 0 == len(jobs_list):
        #     return True
        #print(len(jobs))

        for jobs in jobs_list:
            #公司职位
            positionname=jobs.get("data-positionname")
            # print("招聘职位:", positionname)
            #公司名称
            company=jobs.get("data-company")
            # print("公司名称:", company)
            #薪资
            salary = jobs.get("data-salary")
            # print("薪资:", salary)

            experience = "未知经验"
            education="未知学历"
            edu_exp_money_list = jobs.select("div.p_bot div.li_b_l")
            if 0 != len(edu_exp_money_list):
                edu_exp_len = len(edu_exp_money_list[0].text.strip("\n").split("\n"))
                if 2 <= edu_exp_len :
                    if 1 == len(edu_exp_money_list[0].text.strip("\n").split("\n")[1].split("/")):
                        education = edu_exp_money_list[0].text.strip("\n").split("\n")[1].split("/")[0]
                    elif 2 == len(edu_exp_money_list[0].text.strip("\n").split("\n")[1].split("/")):
                        education = edu_exp_money_list[0].text.strip("\n").split("\n")[1].split("/")[0]
                        experience = edu_exp_money_list[0].text.strip("\n").split("\n")[1].split("/")[1].strip(" ")
            # print("经验:", experience)
            # print("学历:", education)


            #公司地址
            address_list = jobs.select("span.add em")
            address = "未知公司地址"
            if 0 != len(address_list):
                address = address_list[0].text
            # print("公司地址:", address)
            #发布时间
            time="1900-01-01"
            time_org=""
            time_new=""
            time_list = jobs.select("span.format-time")
            if 0 != len(time_list):
                time_org=time_list[0].text
                time_new=get_date_time(time_org)
                time=time_new
            #print("发布时间:",time, time_org)

            #公司类型
            com_type="未知公司类型"
            com_type_list=jobs.select("div.industry")
            if 0 != len(com_type_list):
                #com_type = com_type_list[0].text.strip(" ").strip("\n").strip(" ")
                com_type = com_type_list[0].text.strip().replace(" ", "")
            # print("公司类型:",com_type)

            #职位类型
            position_type = jobs.select("div.list_item_bot div.li_b_l span")
            real_pos_type=[ type.text for type in position_type]
            real_position_type=",".join(real_pos_type)
            # print("职位类型:", real_position_type)

            #公司文化
            com_culture="公司文化"
            com_culture_list=jobs.select("div.li_b_r")
            if 0 != len(com_culture_list):
                com_culture = com_culture_list[0].text.strip('“”')
            # print("公司文化:", com_culture)

            record = [positionname, address, company, com_type, salary, education, experience,time, real_position_type, com_culture ]
            # print("\t".join(record))
            cursor.execute(lagou_insert_sql, record)
            con.commit()


def main():
    i = 0
    while True and i < len(real_links_list):
        link = real_links_list[i]
        print("正在获取:", link, ":", (i+1))
        try:
            if True == get_job_information(link):
                print("已经完成:", link, (i + 1))
        except:
            print("获取：", link, "出错")
        else:
            print("--------------------------------")
        finally:
            i += 1

main()



